#ifndef _ATOMIC_OP_ASM_H_
#define	_ATOMIC_OP_ASM_H_

#include <machine/asm.h>

#define	ATOMIC_OP8(OP, INSN)						\
ENTRY_NP(_atomic_##OP##_8)						;\
	mov	x4, x0							;\
1:	ldxrb	w0, [x4]		/* load old value */		;\
	INSN	w2, w1, w0		/* calculate new value */	;\
	stxrb	w3, w2, [x4]		/* try to store */		;\
	cbnz	w3, 1b			/*   succeed? no, try again */	;\
	dmb	st							;\
	ret				/* return old value */		;\
END(_atomic_##OP##_8)

#define	ATOMIC_OP8_NV(OP, INSN)						\
ENTRY_NP(_atomic_##OP##_8_nv)						;\
	mov	x4, x0			/* need r0 for return value */	;\
1:	ldxrb	w0, [x4]		/* load old value */		;\
	INSN	w2, w1, w0		/* calc new (return) value */	;\
	stxrb	w3, w2, [x4]		/* try to store */		;\
	cbnz	w3, 1b			/*   succeed? no, try again */	;\
	dmb	st							;\
	ret				/* return new value */		;\
END(_atomic_##OP##_8_nv)

#define	ATOMIC_OP16(OP, INSN)						\
ENTRY_NP(_atomic_##OP##_16)						;\
	mov	x4, x0							;\
1:	ldxrh	w0, [x4]		/* load old value */		;\
	INSN	w2, w1, w0		/* calculate new value */	;\
	stxrh	w3, w2, [x4]		/* try to store */		;\
	cbnz	w3, 1b			/*   succeed? no, try again */	;\
	dmb	st							;\
	ret				/* return old value */		;\
END(_atomic_##OP##_16)

#define	ATOMIC_OP16_NV(OP, INSN)					\
ENTRY_NP(_atomic_##OP##_16_nv)						;\
	mov	x4, x0			/* need r0 for return value */	;\
1:	ldxrh	w0, [x4]		/* load old value */		;\
	INSN	w2, w1, w0		/* calc new (return) value */	;\
	stxrh	w3, w2, [x4]		/* try to store */		;\
	cbnz	w3, 1b			/*   succeed? no, try again */	;\
	dmb	st							;\
	ret				/* return new value */		;\
END(_atomic_##OP##_16_nv)

#define	ATOMIC_OP32(OP, INSN)						\
ENTRY_NP(_atomic_##OP##_32)						;\
	mov	x4, x0							;\
1:	ldxr	w0, [x4]		/* load old value */		;\
	INSN	w2, w1, w0		/* calculate new value */	;\
	stxr	w3, w2, [x4]		/* try to store */		;\
	cbnz	w3, 1b			/*   succeed? no, try again */	;\
	dmb	st							;\
	ret				/* return old value */		;\
END(_atomic_##OP##_32)

#define	ATOMIC_OP32_NV(OP, INSN)					\
ENTRY_NP(_atomic_##OP##_32_nv)						;\
	mov	x4, x0			/* need r0 for return value */	;\
1:	ldxr	w0, [x4]		/* load old value */		;\
	INSN	w0, w0, w1		/* calc new (return) value */	;\
	stxr	w3, w0, [x4]		/* try to store */		;\
	cbnz	w3, 1b			/*   succeed? no, try again? */	;\
	dmb	sy							;\
	ret				/* return new value */		;\
END(_atomic_##OP##_32_nv)

#define	ATOMIC_OP64(OP, INSN)						\
ENTRY_NP(_atomic_##OP##_64)						;\
	mov	x4, x0							;\
1:	ldxr	x0, [x4]		/* load old value */		;\
	INSN	x2, x1, x0		/* calculate new value */	;\
	stxr	w3, x2, [x4]		/* try to store */		;\
	cbnz	w3, 1b			/*   succeed? no, try again */	;\
	dmb	st							;\
	ret				/* return old value */		;\
END(_atomic_##OP##_64)

#define	ATOMIC_OP64_NV(OP, INSN)					\
ENTRY_NP(_atomic_##OP##_64_nv)						;\
	mov	x4, x0			/* need r0 for return value */	;\
1:	ldxr	x0, [x4]		/* load old value */		;\
	INSN	x0, x0, x1		/* calc new (return) value */	;\
	stxr	w3, x0, [x4]		/* try to store */		;\
	cbnz	w3, 1b			/*   succeed? no, try again? */	;\
	dmb	sy							;\
	ret				/* return new value */		;\
END(_atomic_##OP##_64_nv)

#if defined(_KERNEL)

#define	ATOMIC_OP_ALIAS(a,s)	STRONG_ALIAS(a,s)

#else /* _KERNEL */

#define	ATOMIC_OP_ALIAS(a,s)	WEAK_ALIAS(a,s)

#endif /* _KERNEL */

#endif /* _ATOMIC_OP_ASM_H_ */
